package com.coderman.task.service;

import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.io.File;
import java.io.IOException;
import java.util.List;

/**
 * Created by ${fanchunshuai} on 2017-7-13.
 *
 * @version: V1.0
 * @Desc: 爬取csdn用户个人信息
 * @Copyright (c) 2017 58到家-版权所有
 */
public class ParseUserExt  implements PageProcessor {
    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private String userLink;
    private File file = new File("E:\\spiderFile\\csdnUserInfo2.txt");
    public String getUserLink() {
        return userLink;
    }

    public void setUserLink(String userLink) {
        this.userLink = userLink;
    }

    @Override
    public void process(Page page) {

        page.addTargetRequests(page.getHtml().links().regex("("+userLink+")").all());
        Html html = page.getHtml();
        //关注数量
        String focusNum = html.xpath("//div[@class='person_info_con']/dl[@class='person-photo']/dd[@class='focus_num']/b/text()").get();
        //粉丝数量
        String fansNum = html.xpath("//div[@class='person_info_con']/dl[@class='person-photo']/dd[@class='fans_num']/b/text()").get();
        //个人信息
        String personDetail = html.xpath("//div[@class='person_info_con']/dl[@class='person-info']/dd[@class='person-detail']/text()").get();
        //关注的用户
        List<String> personLinks = html.xpath("//div[@class='persion_article']/div[@class='mod_relations']//a/@href").all();
        //熟悉的领域
        //List<String> divDetail = html.xpath("//div[@id='divDetail']/div[@class='myDetails activeContent']/div[@class='mod_field_skill']").all();
        //System.out.println(divDetail);
        //昵称
        String nickName = html.xpath("//div[@class='person_info_con']/dl[@class='person-info']/dt[@class='person-nick-name']/span/text()").get();

        String commentinfo = html.xpath("//div[@class='person_info_con']/dl[@class='person-info']/dd[@class='person-sign']/text()").get();

       /* System.out.println(focusNum);
        System.out.println(fansNum);
        System.out.println(personDetail);
        System.out.println(personLinks);
        System.out.println(nickName+"-----------------");
        System.out.println(commentinfo);*/
        StringBuffer buffer = new StringBuffer();
        buffer.append(userLink+":");
        buffer.append(nickName+":");
        buffer.append(focusNum+":");
        buffer.append(fansNum+":");
        buffer.append(personLinks+"\n");

        System.out.printf(buffer.toString());
        try {
            Files.append(buffer.toString(),file, Charsets.UTF_8);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {

        File file = new File("E:\\spiderFile\\csdnUserLink2.txt");
        try {
            List<String> list = FileUtils.readLines(file,"utf-8");
            for (String urls : list){

                String [] urlArray = urls.split(",");
                for (String url : urlArray){
                    if(StringUtils.isBlank(url)){
                        continue;
                    }
                    ParseUserExt parseUserExt = new ParseUserExt();
                    parseUserExt.setUserLink(url.trim());
                    Spider.create(parseUserExt).addPipeline(new ConsolePipeline())
                            .addUrl(parseUserExt.getUserLink())
                            .thread(1).run();
                }

            }


        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void exeParse(String url){
        ParseUserExt parseUserExt = new ParseUserExt();
        parseUserExt.setUserLink(url.trim());
        Spider.create(parseUserExt).addPipeline(new ConsolePipeline())
                .addUrl(parseUserExt.getUserLink())
                .thread(1).run();
    }


}


